package com.zxl.grocery.microservice.crawler.processor;

import com.zxl.grocery.microservice.common.util.ListUtils;
import com.zxl.grocery.microservice.common.util.StrKit;
import com.zxl.grocery.microservice.common.util.ToolUtil;
import com.zxl.grocery.microservice.crawler.common.base.AbstractPageProcessor;
import com.zxl.grocery.microservice.crawler.common.utils.SpringContextHolder;
import com.zxl.grocery.microservice.crawler.entity.CrawlerNovel;
import com.zxl.grocery.microservice.crawler.mapper.CrawlerNovelMapper;
import com.zxl.grocery.microservice.crawler.pipeline.NovelPipeline;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;

import javax.annotation.Resource;
import java.util.List;


/**
 * info:笔趣阁爬虫程序
 * Created by zxl on 2017/8/8 0008.
 */
@Component
public class BiQuGeProcessor extends AbstractPageProcessor {

    public static final String shengXuUrl = "http://www.biqiuge.com/book/4772/";//圣墟列表页
    public static final String SX = "http://www.biqiuge.com/book/4772/\\d+.html";//圣墟详细页

    public static final String yiNianYongHengUrl = "http://www.biqiuge.com/book/630/";//一念永恒列表页
    public static final String YNYH = "http://www.biqiuge.com/book/630/\\d+.html";//一念永恒详细页

    public static final String woShiZhiZunUrl = "http://www.biqiuge.com/book/2956/";//我是至尊列表页
    public static final String WSZZ = "http://www.biqiuge.com/book/2956/\\d+.html";//我是至尊详细页

    private String sxAuthor = null;
    private String sxTitle = null;
    private String ynyhAuthor = null;
    private String ynyhTitle = null;
    private String wszzAuthor = null;
    private String wszzTitle = null;

    @Resource
    private CrawlerNovelMapper crawlerNovelMapper = SpringContextHolder.getBean(CrawlerNovelMapper.class);

    @Autowired
    private NovelPipeline novelPipeline;
    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
//    private Site site = Site.me()
//            .setDomain("http://www.biqiuge.com")
//            .setSleepTime(3000)
//            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
    @Override
    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
    public void process(Page page) {
        System.out.println(page.getUrl());
        if(page.getUrl().regex(shengXuUrl).match()){
            if (!page.getUrl().regex(SX).match()) {
                sxAuthor = page.getHtml().xpath("//*[@id=\"info\"]/p[1]/a/text()").toString();
                sxTitle = page.getHtml().xpath("//*[@id=\"info\"]/h1/text()").toString();
//                System.out.println("title:"+ title + "author:"+author);
                List<String> sxUrls = page.getHtml().xpath("//div[@id='list']").links().regex(SX).all();
                List<String> sxExistLinks = crawlerNovelMapper.getAllLinksByTypeId(1);
                List<String> sxNeedAddUrls = ListUtils.getDiffrentLists(sxUrls,sxExistLinks);
                System.out.println("圣墟新增的urls:"+sxNeedAddUrls.size());
                page.addTargetRequests(sxNeedAddUrls);
            } else {
                String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
                String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
                String link = page.getUrl().toString();
                Integer chapter_sort = ToolUtil.getChapterSort(chapter);
                System.out.println("chapter:::"+chapter);
                System.out.println(content);
                if(content != null){
                    if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新圣墟最新章节！")){
                        content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新圣墟最新章节！",""));
                        System.out.println(content);
                    }
                }
                if (StringUtils.isNotEmpty(content)) {
                    CrawlerNovel novel = new CrawlerNovel();
                    novel.setTitle(sxTitle);
                    novel.setNovelist(sxAuthor);
                    novel.setChapterSort(chapter_sort);
                    novel.setChapter(chapter);
                    novel.setContent(content);
                    novel.setLink(link);
                    novel.setTypeId(1);
                    novel.setSourcesId(8);
                    page.putField("novel", novel);
                }
            }
        }
        System.out.println(page.getUrl());
        if(page.getUrl().regex(yiNianYongHengUrl).match()){

            if(!page.getUrl().regex(YNYH).match()){
                ynyhAuthor = page.getHtml().xpath("//*[@id=\"info\"]/p[1]/a/text()").toString();
                ynyhTitle = page.getHtml().xpath("//*[@id=\"info\"]/h1/text()").toString();
                List<String> ynyhUrls = page.getHtml().xpath("//div[@id='list']").links().regex(YNYH).all();
                List<String> ynyhExistLinks = crawlerNovelMapper.getAllLinksByTypeId(3);
                List<String> ynyhNeedAddUrls = ListUtils.getDiffrentLists(ynyhUrls,ynyhExistLinks);
                System.out.println("一念永恒新增的urls:"+ynyhNeedAddUrls.size());
                page.addTargetRequests(ynyhNeedAddUrls);
            } else {
                String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
                String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
                String link = page.getUrl().toString();
                Integer chapter_sort = ToolUtil.getChapterSort(chapter);
                System.out.println("chapter:::"+chapter);
                System.out.println(content);
                if(content != null){
                    if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！")){
                        content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！",""));
                        System.out.println(content);
                    }
                }
                if (StringUtils.isNotEmpty(content)) {
                    CrawlerNovel novel = new CrawlerNovel();
                    novel.setTitle(ynyhTitle);
                    novel.setNovelist(ynyhAuthor);
                    novel.setChapterSort(chapter_sort);
                    novel.setChapter(chapter);
                    novel.setContent(content);
                    novel.setLink(link);
                    novel.setTypeId(3);
                    novel.setSourcesId(8);
                    page.putField("novel", novel);
                }
            }
        }
        System.out.println(page.getUrl());
        if(page.getUrl().regex(woShiZhiZunUrl).match()){

            if(!page.getUrl().regex(WSZZ).match()){
                wszzAuthor = page.getHtml().xpath("//*[@id=\"info\"]/p[1]/a/text()").toString();
                wszzTitle = page.getHtml().xpath("//*[@id=\"info\"]/h1/text()").toString();
                List<String> ynyhUrls = page.getHtml().xpath("//div[@id='list']").links().regex(WSZZ).all();
                System.out.println("----------------++++++++++++++++++++++++++++");
                List<String> ynyhExistLinks = crawlerNovelMapper.getAllLinksByTypeId(4);
                System.out.println("------------------------------------------");
                System.out.println(ynyhUrls.size() + ynyhExistLinks.size());
                List<String> ynyhNeedAddUrls = ListUtils.getDiffrentLists(ynyhUrls,ynyhExistLinks);
                System.out.println("我是至尊新增的urls:"+ynyhNeedAddUrls.size());
                page.addTargetRequests(ynyhNeedAddUrls);
            } else {
                String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
                String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
                String link = page.getUrl().toString();
                Integer chapter_sort = ToolUtil.getChapterSort(chapter);
                System.out.println("chapter:::"+chapter);
                System.out.println(content);
                if(content != null){
                    if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新我是至尊最新章节！")){
                        content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新我是至尊最新章节！",""));
                        System.out.println(content);
                    }
                }
                if (StringUtils.isNotEmpty(content)) {
                    CrawlerNovel novel = new CrawlerNovel();
                    novel.setTitle(wszzTitle);
                    novel.setNovelist(wszzAuthor);
                    novel.setChapterSort(chapter_sort);
                    novel.setChapter(chapter);
                    novel.setContent(content);
                    novel.setLink(link);
                    novel.setTypeId(4);
                    novel.setSourcesId(8);
                    page.putField("novel", novel);
                }
            }
        }
    }


    @Override
    public String[] initTargetUrl() {
        String[] targetUrls = new String[]{shengXuUrl,yiNianYongHengUrl,woShiZhiZunUrl};
        return targetUrls;
    }

//    public void main(String[] args) {
//        BiQuGeProcessor biQuGeProcessor=new BiQuGeProcessor();
//        biQuGeProcessor.setPipelines(Arrays.asList(novelPipeline));//这种方法生成的list，是不支持添加或删除元素的
//        biQuGeProcessor.start();
//    }

//    @Override
//    public Site getSite() {
//        return site;
//    }
}
